﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using LogisticsCrawler.Core;
using System.Threading;
using System.Text.RegularExpressions;
using LogisticsCrawler.Core.RegularParse;

namespace LogisticsCrawler
{
    public partial class FrmCrawler : Form
    {
        /// <summary>
        /// 提供在各种同步模型中传播同步上下文的基本功能
        /// </summary>
        public static SynchronizationContext SynchronizationContext { get; set; }
        private const string WEB_DOMAIN = "http://www.701sou.com";
        private string BASE_URI = "http://www.701sou.com/index.php?lang=zhcn&act=approximatch%20search&where=北京市&dAction=Search&sort=";
        private Uri _uri;
        private static int count = 0;       //总记录数
        private static int cursor = 0;      //当前记录
        public static string _mainType = "";
        public static int _maxNum = 0;
        public static int _start = 0;

        private void btCrawler_Click(object sender, EventArgs e)
        {
            if (this.comboBox1.SelectedIndex < 0)
            {
                MessageBox.Show("请选择一个抓取行业类型");
                return;
            }
            //主类型
            _mainType = comboBox1.SelectedItem.ToString();
            _maxNum = Convert.ToInt32(txtMaxnum.Text);
            _start = Convert.ToInt32(txtStart.Text);
            //最大记录号
            for (int i = _start; i <= _maxNum; i ++)
            {
                _uri = new Uri(BASE_URI + "&what="+_mainType + "&item=" + ((i-1)*10+1).ToString());

                PageSniff sniff = new PageSniff(_uri, Done);
                Thread.Sleep(100);
            }
        }

        /// <summary>
        /// 完成事件
        /// </summary>
        public void Done(string html)
        {
            IList<CrawlerCompany> list = getContent(html);            

            if (list != null && list.Count > 0)
            {
                foreach (CrawlerCompany c in list)
                {
                    if (!string.IsNullOrEmpty(c.Fromurl))
                    {
                        PageSniff sniff = new PageSniff(new Uri(c.Fromurl), c, SubPageDone);
                    }
                    else
                    {
                        SubPageDoneV2(c);   
                    }
                    cursor++;
                    SynchronizationContext = SynchronizationContext.Current;
                    SynchronizationContext.Post(
                        delegate
                        {
                            lblCount.Text = "记录数：" + cursor.ToString();
                        }, null);
                }
            }           
        }

        /// <summary>
        /// 二级页完成事件
        /// </summary>
        #region void SubPageDone(CrawlerCompany company,string html)
        public void SubPageDone(CrawlerCompany company,string html)
        {
            getSubinfo(ref company,html);

            SubPageDoneV2(company);
        }

        public void SubPageDoneV2(CrawlerCompany company)
        {
            SynchronizationContext = SynchronizationContext.Current;
            SynchronizationContext.Post(
                delegate
                {
                    StringBuilder sb = new StringBuilder();
                    using (ManagerCrawlerCompany mcc = new ManagerCrawlerCompany())
                    {
                        if (mcc.Insert(company))
                        {
                            //count++;
                            //lblCount.Text = "记录数：" + count.ToString();
                            sb.AppendLine("记录插入成功...");
                        }
                        else
                            sb.AppendLine(ManagerFactory.ManagerCrawlerCompany.Message);
                    }

                    sb.AppendLine(company.Name + " " + company.Fromurl);
                    sb.AppendLine("地址：" + company.Address);
                    sb.AppendLine("电话：" + company.Phone);
                    sb.AppendLine("分类：" + company.Category);
                    sb.AppendLine("网址：" + company.Website);
                    sb.AppendLine();

                    txtResult.Text = sb.ToString();
                }, null
                );
        }
        #endregion

        // -------------- function ----------------- //
        /// <summary>
        /// 获取抓取公司信息集合
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        #region static IList<CrawlerCompany> getContent(string html)
        public static IList<CrawlerCompany> getContent(string html)
        {
            IList<CrawlerCompany> list = new List<CrawlerCompany>();
            try
            {                
                //抓有链接的
                string pattern =
                    "<div\\s+class=\"name\">\\s+<span[^>]+>\\d*</span>\\s+<span><a\\s+href=\"(?<fromurl>\\S*)\"\\s+onclick=\"loadBusinessInfo\\(\\'\\d+\\',\\s+\\'\\d+\\'\\);\\s+return\\s+false;\"\\s+class=\"businessName\">(?<name>.*?)</a></span><span\\s+id=\"phoneSpan\\d+\"></span><script\\stype=\"text/javascript\">(?<script>[^<]*)</script>\\s+</div>";
                pattern += "[^<]*<div\\s+class=\"checkbox\"><input\\s+type=\"checkbox\"[^>]*></div>\\s+</div>";
                pattern += "[^<]*<div\\s+class=\"address\">(?<address>[^<]*)<br\\s+/>\\s+<span\\s+style=\"line-height: 30px;\">分类:\\s+(?<category>[^<]+)</span>";
                Regex r = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                MatchCollection mc = r.Matches(html);//满足pattern的匹配集合
                if (mc.Count > 0)
                {
                    foreach (Match match in mc)
                    {
                        GroupCollection gc = match.Groups;
                        CrawlerCompany c = new CrawlerCompany();
                        //phone
                        string script = gc["script"].Value;
                        c.MainType = _mainType;
                        c.Phone = getPhone(script).Replace("\r\n", "").Trim(); 
                        //address
                        c.Address = gc["address"].Value.Replace("\r\n","").Trim();                        
                        //category
                        c.Category = gc["category"].Value;
                        //url
                        c.Fromurl = gc["fromurl"]!=null ? WEB_DOMAIN + gc["fromurl"].Value : string.Empty;
                        //name
                        c.Name = gc["name"].Value;
                        
                        list.Add(c);
                    }
                }
                //抓取没有链接的
                pattern =
                    "<div\\s+class=\"name\">\\s+<span[^>]+>\\d*</span>\\s+<span>(?<name>.*?)</span><span\\s+id=\"phoneSpan\\d+\"></span><script\\stype=\"text/javascript\">(?<script>[^<]*)</script>\\s+</div>";
                pattern += "[^<]*<div\\s+class=\"checkbox\"><input\\s+type=\"checkbox\"[^>]*></div>\\s+</div>";
                pattern += "[^<]*<div\\s+class=\"address\">(?<address>[^<]*)<br\\s+/>\\s+<span\\s+style=\"line-height: 30px;\">分类:\\s+(?<category>[^<]+)</span>";
                r = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                mc = r.Matches(html);//满足pattern的匹配集合
                if (mc.Count > 0)
                {
                    foreach (Match match in mc)
                    {
                        GroupCollection gc = match.Groups;
                        CrawlerCompany c = new CrawlerCompany();
                        //phone
                        string script = gc["script"].Value;
                        c.MainType = _mainType;
                        c.Phone = getPhone(script).Replace("\r\n", "").Trim();
                        //address
                        c.Address = gc["address"].Value.Replace("\r\n", "").Trim();
                        //category
                        c.Category = gc["category"].Value;
                        //url
                        c.Fromurl = string.Empty;
                        //name
                        string tempName = gc["name"].Value;                        
                        if(tempName.StartsWith("<a"))
                        {
                            int indexAend = tempName.IndexOf(">");
                            int indexB = tempName.IndexOf("</a>");
                            c.Name = tempName.Substring(indexAend+1, indexB - indexAend-1);
                        }
                        else
                            c.Name = tempName;

                        list.Add(c);
                    }
                }
            }
            catch { }
            return list;
        }
        #endregion

        /// <summary>
        /// 根据JS获取电话
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        #region static string getPhone(string html)
        public static string getPhone(string html)
        {
            const string STR_FROM = "d.innerHTML+=";
            StringBuilder sb = new StringBuilder();

            int startIndex = 0;
            int index = html.IndexOf(STR_FROM,startIndex);
            string temp = string.Empty;
            while (index > -1)
            {
                //抓取语句
                startIndex = index + STR_FROM.Length;
                index = html.IndexOf(STR_FROM, startIndex);
                if (index > -1)
                    temp = html.Substring(startIndex, index - startIndex);
                else
                    temp = html.Substring(startIndex, html.Length-startIndex);
                temp = temp.Replace(";", "").Replace("String.fromCharCode(", "").Replace(")", "");
                //分析语句
                if (temp.IndexOf("+") > -1)
                {
                    //如果是String.fromCharCode(x+y)形式
                    string[] num = temp.Split('+');
                    int numFromChar = Convert.ToInt32(num[0]) + Convert.ToInt32(num[1]) - 48;
                    sb.Append(numFromChar.ToString());
                }
                else if (temp.IndexOf("?") > -1)
                {
                    //如果是d.innerHTML+=1?'1':'7';
                    if (temp.StartsWith("1"))
                        sb.Append(temp[3].ToString());
                    else
                        sb.Append(temp[7].ToString());
                }
                else
                {
                    //直接是数字d.innerHTML+="6";
                    sb.Append(temp.ToString());
                }
            }

            return sb.ToString().Replace("'","");
        }
        #endregion

        /// <summary>
        /// 获取二级页面的website、phone、services、companyInfo
        /// </summary>
        /// <param name="html"></param>
        #region void getSubinfo(ref CrawlerCompany company,string html)
        public void getSubinfo(ref CrawlerCompany company,string html)
        {
            try
            {
                string pattern =
                    "网站:</strong>\\s+<a\\s+href=\"(?<href>.*?)\"[^>]*>.*?</a></span></span>\\s+</div>";
                Regex r = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                Match mc = r.Match(html);
                if (mc.Success)
                {
                    GroupCollection gc = mc.Groups;                    
                    company.Website = gc["href"].Value;
                }
                pattern =
                    "<h3[^>]*>公司简介:</h3>\\s+(?<intro>[\\s*\\S*]*)</div>";
                r = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                mc = r.Match(html);
                if (mc.Success)
                {
                    GroupCollection gc = mc.Groups;
                    string res = gc["intro"].Value;
                    int index =res.IndexOf("</div>");
                    if (index > -1)
                    {
                        company.Intro = res.Substring(0, index).Replace("<br />","\r\n");
                    }
                }                
                pattern =
                   "<h3[^>]*>产品与服务:</h3>\\s+(?<service>[\\s*\\S*]*)</div>";
                r = new Regex(pattern, RegexOptions.IgnoreCase | RegexOptions.Compiled);
                mc = r.Match(html);
                if (mc.Success)
                {
                    GroupCollection gc = mc.Groups;
                    string res = gc["service"].Value.Replace("<ul>", "").Replace("</ul>", "").Replace("<li>", "").Replace("</li>", "");
                    int index = res.IndexOf("</div>");
                    if (index > -1)
                    {
                        company.Service = res.Substring(0, index);
                    }
                }
            }
            catch { }
        }
        #endregion

        #region constructor
        public FrmCrawler()
        {
            InitializeComponent();
        }
        #endregion
    }
}
